package com.waijiaojun.tpo.service;

import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.waijiaojun.tpo.crawler.CategoryRefScenarioDto;
import com.waijiaojun.tpo.crawler.CrawlerTopUrls;
import com.waijiaojun.tpo.crawler.PoiCategoryUtil;
import com.waijiaojun.tpo.crawler.TpoXzDto;
import com.waijiaojun.tpo.entity.*;
import com.waijiaojun.tpo.repository.*;
import com.waijiaojun.tpo.rest.RestServerException;
import com.waijiaojun.tpo.web.ExerciseController;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.RequestBody;

import javax.annotation.Resource;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Service
public class CrawlerXiaoZhanTpoService {

    private static Logger logger = LoggerFactory.getLogger(ExerciseController.class);

    @Resource
    private ScenarioDao scenarioDao;
    @Resource
    private VocabularyDao vocabularyDao;
    @Resource
    private QuestionDao questionDao;
    @Resource
    private SentenceDao sentenceDao;

    public void crawlerData(String type) {
        try {
            List<TpoXzDto> list = null;
//            List<TpoXzDto> list = CrawlerTopUrls.getTpoXzDtos();
//            List<TpoXzDto> list = CrawlerTopUrls.getExtraDtos();
//            List<TpoXzDto> list = CrawlerTopUrls.getOgDtos();
//            List<TpoXzDto> list = genTestList(); // 测试
            if(type.equals("Tpo")){
                list = CrawlerTopUrls.getTpoXzDtos();
            }else if(type.equals("Extra")){
                list = CrawlerTopUrls.getExtraDtos();
            }else if(type.equals("Og")){
                list = CrawlerTopUrls.getOgDtos();
            }

            for (int i = 0; i < list.size(); i++) {
                logger.info("\n\n\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> \n 正在爬取第" + (i + 1) + "篇文章(总计" + list.size() + ")---->>" + list.get(i).toString());

                prepareHtml(list.get(i));
                logger.info("\n\n第" + (i + 1) + "篇文章爬完(总计" + list.size() + "). <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<");
            }
        } catch (Exception e) {
            logger.error(e.getMessage(), e);
            throw new RestServerException(e.getMessage());
        }
    }

    public void updateArticle(String type) {
        try {
//            List<TpoXzDto> list = genTestList(); // 测试
//            List<TpoXzDto> list = CrawlerTopUrls.getTpoXzDtos();
//            List<TpoXzDto> list = CrawlerTopUrls.getExtraDtos();
//            List<TpoXzDto> list = CrawlerTopUrls.getOgDtos();

            List<TpoXzDto> list = null;
            if(type.equals("Tpo")){
                list = CrawlerTopUrls.getTpoXzDtos();
            }else if(type.equals("Extra")){
                list = CrawlerTopUrls.getExtraDtos();
            }else if(type.equals("Og")){
                list = CrawlerTopUrls.getOgDtos();
            }

            for (int i = 0; i < list.size(); i++) {
                logger.info("\n\n\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> \n 正在爬取第" + (i + 1) + "篇文章(总计" + list.size() + ")---->>" + list.get(i).toString());

                prepareHtmlForUpdateArticle(list.get(i));
                logger.info("\n\n第" + (i + 1) + "篇文章爬完(总计" + list.size() + "). <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<");
            }
        } catch (Exception e) {
            logger.error(e.getMessage(), e);
            throw new RestServerException(e.getMessage());
        }
    }

    private List<TpoXzDto> genTestList() {
        List<TpoXzDto> list = new ArrayList<>();
        list.add(new TpoXzDto("OG3.6", "http://top.zhan.com/toefl/listen/review-891-13.html", "https://static2.zhan.com/beikao/images/listen/53-1.jpg?v=201802071030"));
        return list;
    }

    private void prepareHtml(TpoXzDto dto) throws IOException {
        try (final WebClient webClient = new WebClient()) {
            final HtmlPage page = webClient.getPage(dto.getUrl()); // 表格题

            // 等待页面执行Javascript。页面的重点词汇通过调用Ajax获取的，然后渲染到页面上
            int wait = 0;
            int nbProcess = 1;
            while (nbProcess > 0 && wait < 10) {
                nbProcess = webClient.waitForBackgroundJavaScript(1000);
                if (wait == 9) {
                    System.err.println("** needs more time ** ");
                }
                wait++;
                print("wait:" + wait + "    nbProcess:" + nbProcess);
            }
            final String pageAsXml = page.asXml();
//            print(pageAsXml);// 打印页面html
//            final String pageAsText = page.asText();
            paresTpoPage(pageAsXml, dto);
        }
    }

    private void prepareHtmlForUpdateArticle(TpoXzDto dto) throws IOException {
        try (final WebClient webClient = new WebClient()) {
            final HtmlPage page = webClient.getPage(dto.getUrl()); // 表格题

            // 等待页面执行Javascript。页面的重点词汇通过调用Ajax获取的，然后渲染到页面上
            int wait = 0;
            int nbProcess = 1;
            while (nbProcess > 0 && wait < 10) {
                nbProcess = webClient.waitForBackgroundJavaScript(1000);
                if (wait == 9) {
                    System.err.println("** needs more time ** ");
                }
                wait++;
                print("wait:" + wait + "    nbProcess:" + nbProcess);
            }
            final String pageAsXml = page.asXml();
//            print(pageAsXml);// 打印页面html
//            final String pageAsText = page.asText();
            paresTpoPageForUpdateArticle(pageAsXml, dto);
        }
    }

    // 保存数据到数据库
    @Transactional
    public void saveDatas(Scenario scenario, List<Vocabulary> vocabularyList, List<Question> questionList) {
        logger.info("\n\n\n开始插入数据到数据库 -------------------------------------------------------------------");
        scenarioDao.insertSelective(scenario);
        logger.info("    scenario_id:" + scenario.getId() + "\n\n");

        if (vocabularyList != null && vocabularyList.size() > 0) {
            for (Vocabulary v : vocabularyList) {
                v.setScenarioId(scenario.getId());
            }
            vocabularyDao.insertList(vocabularyList);
        }

        for (Question v : questionList) {
            v.setScenario_id(scenario.getId());
        }
        questionDao.insertList(questionList);

    }

    private void paresTpoPage(String html, TpoXzDto dto) throws IOException {
        Scenario scenario = new Scenario();// 场景
        List<Vocabulary> vocabularyList = new ArrayList<>();// 词汇
        List<Question> questionList = new ArrayList<>();// 问题

        scenario.setTpo_name(dto.getTpoName());
        scenario.setCoverUrl(dto.getImage_url());

        //解析Url获取Document对象
//        Document document = Jsoup.connect(url).get();
        Document document = Jsoup.parse(html);
        //获取网页源码文本内容
        String doc = document.toString();
        // 按指定模式在字符串查找
        String pattern = "https:[\\w:/ \\.\\-\\_]+mp3";

        // 创建 Pattern 对象
        Pattern r = Pattern.compile(pattern);

        // 现在创建 matcher 对象
        Matcher m = r.matcher(doc);
        if (m.find()) {
            String autio_url = m.group(0);
            scenario.setAudioUrl(autio_url.trim());
            print("MATCH audio_url:" + autio_url);
        } else {
            print("NO MATCH");
        }
//            print(document.toString());

        // TOP Name
        Elements topElem = document.select(".main_title_review");
        String tpoName_titme = topElem.text();
        String[] strs = tpoName_titme.split(" ");
        String tpoName = strs[0];
        String nameCn = strs[1];
        scenario.setNameCn(nameCn);
        print("TPO Name:" + tpoName);
        print("Article Title Cn:" + nameCn);

        Elements nameEnElem = document.select(".sub_title_review");
        String nameEn = nameEnElem.text();
        scenario.setNameEn(nameEn);
        print("Article Title En:" + nameEn);

        Elements article = document.getElementsByClass("article");
        // 英文
        Elements articleEnList = article.select("span .text");
        StringBuffer articleEnBuf = new StringBuffer("");
        for (int i = 0; i < articleEnList.size(); i++) {
            String en = articleEnList.get(i).text();
            articleEnBuf.append(en.trim());
            print(i + ". " + en.trim());
        }
        scenario.setArticleEn(articleEnBuf.toString());

        // 中文
        Elements articleCnList = article.select("span[data-translation]");
        StringBuffer articleCnBuf = new StringBuffer("");
        for (int i = 0; i < articleCnList.size(); i++) {
            String cn = articleCnList.get(i).attr("data-translation");
            articleCnBuf.append(cn.trim());
            print(i + ". " + cn.trim());
        }
        scenario.setArticleCn(articleCnBuf.toString());

        // 词汇 .toefl_listen_review_left_content .article .underline
        Elements vocabularys = document.select(".toefl_listen_review_left_content .article .underline");
        print("词汇:");
        long n = 0L;
        for (int i = 0; i < vocabularys.size(); i++) {
            String vocabulary = vocabularys.get(i).text();
            Vocabulary v = new Vocabulary();
            v.setWord(vocabulary);
            // 词汇去重
            if (!containsVocabularys(vocabularyList, v)) {
                vocabularyList.add(v);
                n++;
            }
            v.setOrder(n);
            print(vocabulary);
        }

        printQuestion(1L, document, questionList);


        // 其他问题对应的连接
        Elements as = document.getElementById("footer_review").getElementsByTag("a");
        for (int i = 1; i < as.size(); i++) {
            String question_href = as.get(i).attr("href");
            Document document_q = Jsoup.connect("http://top.zhan.com" + question_href).get();
            printQuestion(i + 1L, document_q, questionList);
        }

        // 保存数据到数据库
        saveDatas(scenario, vocabularyList, questionList);

    }

    private void paresTpoPageForUpdateArticle(String html, TpoXzDto dto) throws IOException {
        Scenario scenario = new Scenario();// 场景
        List<Vocabulary> vocabularyList = new ArrayList<>();// 词汇
        List<Question> questionList = new ArrayList<>();// 问题

        scenario.setTpo_name(dto.getTpoName());

        //解析Url获取Document对象
        Document document = Jsoup.parse(html);

        // TOP Name
        Elements topElem = document.select(".main_title_review");
        String tpoName_titme = topElem.text();
        String[] strs = tpoName_titme.split(" ");
        String tpoName = strs[0];
        String nameCn = strs[1];
        print("TPO Name:" + tpoName);
        print("Article Title Cn:" + nameCn);

        Elements nameEnElem = document.select(".sub_title_review");
        String nameEn = nameEnElem.text();
        print("Article Title En:" + nameEn);

        Elements article = document.getElementsByClass("article");
        // 英文
        Elements articleEnList = article.select("span .text");
        StringBuffer articleBuf = new StringBuffer("");
        List<String> enList = new ArrayList<>();
        for (int i = 0; i < articleEnList.size(); i++) {
            String en = articleEnList.get(i).text().trim();
            enList.add(en);
        }

        // 中文
        List<String> cnList = new ArrayList<>();
        Elements articleCnList = article.select("span[data-translation]");
        for (int i = 0; i < articleCnList.size(); i++) {
            String cn = articleCnList.get(i).attr("data-translation").trim();
            cnList.add(cn);
        }

        // 保存句子
        List<Sentence> sentenceList = new ArrayList<>();
        Long scenarioId = scenarioDao.findIdByTpoName(scenario.getTpo_name());
        for(int i=0;i<enList.size();i++){
            String en = enList.get(i);
            String cn = cnList.get(i);
            // 中英文各一句
            articleBuf.append("<p>").append(en).append("</p>").append("<p>").append(cn).append("</p>");
            Sentence sentence = new Sentence();
            sentence.setOrder(i+1);
            sentence.setTextCn(cn);
            sentence.setTextEn(en);
            sentence.setScenarioId(scenarioId);
            sentenceList.add(sentence);
        }
        print(articleBuf.toString());
        scenario.setArticleEn(articleBuf.toString());
        // 保存文章
        scenarioDao.updateByTpoName(scenario);

        // 保存文章中的每一个句子
        sentenceDao.deleteByScenarioId(scenarioId);
        sentenceDao.insertList(sentenceList);
    }

    private boolean containsVocabularys(List<Vocabulary> vocabularyList, Vocabulary v) {
        if (vocabularyList == null) {
            return false;
        }
        for (Vocabulary vo : vocabularyList) {
            if (vo.getWord().equals(v.getWord())) {
                return true;
            }
        }
        return false;
    }

    private void printQuestion(Long number, Document document, List<Question> questionList) {
        Question questionObj = new Question();
        questionList.add(questionObj);
        Long type = null;
        // 问题
        Elements question = document.select("div .question");

        // 题干
        String questionContent = question.select("div .q_tit").select("div[class='left text']").text();
        print("Question " + number + " : " + questionContent);
        questionObj.setContent_en(questionContent);
        questionObj.setNumber(number);

        // 选项
        StringBuffer optionsSb = new StringBuffer("[");
        Elements table = question.select("table");
        Elements sort = question.select(" .dragsec");
        // 表格题 选项
        if (table.size() > 0) {
            optionsSb.append("{");
            StringBuffer columns = new StringBuffer("\"columns\": [");
            StringBuffer rows = new StringBuffer("\"rows\": [");
            Elements trs = table.select("tr");
            for (int i = 0; i < trs.size(); i++) {
                if (i == 0) {
                    // 第一行提取列头
                    Elements tds = trs.get(i).select("td");
                    for (int j = 0; j < tds.size(); j++) {
                        if (j != 0) {
                            String columnHead = tds.get(j).text();
                            columnHead = chansformSpecialChar(columnHead);
                            print("column " + j + ". " + columnHead);
                            columns.append("{\"id\": " + j + ",\"value\": \"" + columnHead + "\"}");
                            if (j != tds.size() - 1) {
                                columns.append(",");
                            }
                        }
                    }
                    columns.append("]");
                } else {
                    // 提取行头
                    Elements tds = trs.get(i).select("td");
                    for (int j = 0; j < tds.size(); j++) {
                        if (j == 0) {
                            String rowHead = tds.get(j).text();
                            rowHead = chansformSpecialChar(rowHead);
                            print("row " + i + ". " + rowHead);
                            rows.append("{\"id\": " + i + ",\"value\": \"" + rowHead + "\"}");
                            if (i != trs.size() - 1) {
                                rows.append(",");
                            }
                        }
                    }
                }

            }
            rows.append("]");
            type = 4L;
            optionsSb.append(rows).append(",").append(columns);
            optionsSb.append("}]");
        } else if (sort.size() > 0) {
            // 排序题选项
            for (int i = 0; i < sort.size(); i++) {
                String option = sort.get(i).text();
                String label = option.substring(0, 1);
                String opt_cont = option.substring(2).trim();
                opt_cont = chansformSpecialChar(opt_cont);
                print(label + "  " + opt_cont);
                optionsSb.append("{\"id\": " + (i + 1) + ", \"rows\": [],\"columns\": [],\"label\": \"" + label + "\",\"content\": \"" + opt_cont + "\"}");
                if (i != sort.size() - 1) {
                    optionsSb.append(",");
                }
            }
            type = 3L;
            optionsSb.append("]");
        } else {
            //  单选、多选选项
            Elements ops = document.select(".question_option .ops > label");
            for (int i = 0; i < ops.size(); i++) {
                String option = ops.get(i).text();
                String label = option.substring(0, 1);
                String opt_cont = option.substring(2).trim();
                opt_cont = chansformSpecialChar(opt_cont);
                print(label + "  " + opt_cont);
                optionsSb.append("{\"id\": " + (i + 1) + ", \"rows\": [],\"columns\": [],\"label\": \"" + label + "\",\"content\": \"" + opt_cont + "\"}");
                if (i != ops.size() - 1) {
                    optionsSb.append(",");
                }
            }
            optionsSb.append("]");
        }
        String options = optionsSb.toString();
        questionObj.setOption(options);
        print(" options:" + options);

        Elements questionAnswerEle = question.select("div .correctAnswer > span");
        // 答案
        String answer = questionAnswerEle.get(0).text();
        print("  answer:" + answer);
        // 根据答案数量判断题型
        if (type == null) {
            if (answer.length() == 1) {
                type = 1L;
            } else {
                type = 2L;
            }
        }
        questionObj.setType(type);
        print("  question Type: " + type);

        StringBuffer answerJson = new StringBuffer("[");

        String[] anstr = answer.split("");
        if (type.intValue() != 4) {
            for (int i = 0; i < anstr.length; i++) {

                answerJson.append(toId(anstr[i]));
                if (i != anstr.length - 1) {
                    answerJson.append(",");
                }
            }
        } else {
            // 组装表格题的json格式答案
            for (int i = 0; i < anstr.length; i++) {
                answerJson.append("{\"id\": " + i + ",\"result\": [" + toId(anstr[i]) + "]}");
                if (i != anstr.length - 1) {
                    answerJson.append(",");
                }
            }
        }

        answerJson.append("]");
        questionObj.setAnswer(answerJson.toString());
        print("  question_answer: " + questionObj.getAnswer());

        // 是否带有音频
        Elements isAudio = question.select("div div.audio_content");
        if (isAudio.size() > 0) {
            print("  have audio: true");
            questionObj.setAudio_url("true");
        }

        // 题目解析
        Elements resolve = question.select("div.resolve_content");
        if (resolve.size() > 0) {
            String resolve_content = resolve.text();
            print("  " + resolve_content);
            questionObj.setAnalysis(resolve_content);
        } else {
            print("  无题目解析");
        }

    }

    // 处理特殊字符
    private String chansformSpecialChar(String opt_cont) {
        if (StringUtils.isNotBlank(opt_cont)) {
            return opt_cont.replace("\"", "\\\"");
        } else {
            return opt_cont;
        }
    }

    private int toId(String str) {
        if (str.contains("A")) {
            return 1;
        } else if (str.contains("B")) {
            return 2;
        } else if (str.contains("C")) {
            return 3;
        } else if (str.contains("D")) {
            return 4;
        } else if (str.contains("E")) {
            return 5;
        } else if (str.contains("F")) {
            return 6;
        } else if (str.contains("G")) {
            return 7;
        } else if (str.contains("H")) {
            return 8;
        } else {
            throw new RuntimeException("无效的答案：" + str);
        }
    }

    private static void print(String str) {
        System.out.println(str);
    }

    // 分类和场景建立关联关系
    public void linkCategoryAndScenario(){
        // 需要配置excel文件位置
        String excelPath = "d://托福TPO话题分类.xlsx";
        List<CategoryRefScenarioDto> list = PoiCategoryUtil.getCateRefScenDto(excelPath);
        if(list!=null){
            for(CategoryRefScenarioDto dto:list) {
                logger.info(dto.toString());
                scenarioDao.updateCategoryId(dto);
            }
        }else{
            logger.info("无关联数据，不更新数据库");
        }
    }

    public static void main(String[] args) throws IOException {
        final String url = "http://www.kekenet.com/menu/201702/484166.shtml";
        try (final WebClient webClient = new WebClient()) {
            final HtmlPage page = webClient.getPage(url);

            // 等待页面执行Javascript。页面的重点词汇通过调用Ajax获取的，然后渲染到页面上
//            int wait = 0;
//            int nbProcess = 1;
//            while (nbProcess > 0 && wait < 10) {
//                nbProcess = webClient.waitForBackgroundJavaScript(1000);
//                if (wait == 9) {
//                    System.err.println("** needs more time ** ");
//                }
//                wait++;
//                print("wait:" + wait + "    nbProcess:" + nbProcess);
//            }
            final String pageAsXml = page.asXml();
            print(pageAsXml);// 打印页面html
        }
    }
}
